{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2e8b098f216b80e5",
   "metadata": {},
   "source": [
    "## 位置编码\n",
    "数学公式\n",
    "$$PE(pos, 2i) = sin(pos / 10000^{2i / d_{model}})$$\n",
    "$$PE(pos, 2i+1) = cos(pos / 10000^{2i / d_{model}})$$\n",
    "\n",
    "PE表示位置编码矩阵，是一个矩阵，pos表示seq中的某个位置，比如一个seq有4个词，那么pos的取值就是[0, 1, 2, 3]，i表示词向量的索引，比如词向量的长度是3，那么i的取值范围就是[0, 1, 2]\n",
    "\n",
    "PE矩阵的行数和输入的seq的长度一样，比如一个seq有4个词，那么PE矩阵的行数就是4\n",
    "PE矩阵的列数和词向量的长度一样，比如词向量的维度是3，那么PE矩阵的列数就是3\n",
    "\n",
    "PE(pos, 2i)表示PE矩阵中pos行的偶数列\n",
    "PE(pos, 2i+1)表示PE矩阵中pos行的奇数列\n",
    "\n",
    "d_model其实是词向量的维度，论文里默认词向量的维度是512，同时d_model也是Q、K、V的输出维度，也是注意力输出的维度。\n",
    "\n",
    "所以PE矩阵就是在描述位置的信息，seq中的pos位置的词对应的位置信息就是PE矩阵的pos行的向量，而且这个向量的长度和词向量的长度一样，所以可以相加，得到一个包含了该词位置信息的词向量"
   ]
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:34:26.501328Z",
     "start_time": "2025-07-09T12:34:26.497193Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "\n",
    "# x表示一个输入的seq\n",
    "x = torch.rand(4, 8)\n",
    "x"
   ],
   "id": "db87434e12b8eac3",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.8978, 0.4811, 0.1846, 0.3993, 0.8489, 0.9601, 0.0934, 0.2663],\n",
       "        [0.3571, 0.3864, 0.5495, 0.3946, 0.3561, 0.3168, 0.5571, 0.4940],\n",
       "        [0.6726, 0.3538, 0.9980, 0.1661, 0.3805, 0.3075, 0.2332, 0.5693],\n",
       "        [0.6692, 0.9721, 0.9179, 0.8068, 0.2978, 0.5638, 0.2280, 0.9227]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:37:03.621930Z",
     "start_time": "2025-07-09T12:37:03.619609Z"
    }
   },
   "cell_type": "code",
   "source": [
    "seq_len = 4  # 序列长度\n",
    "d_model = 8  # 词向量长度"
   ],
   "id": "16f1a7d120fdeef8",
   "outputs": [],
   "execution_count": 22
  },
  {
   "cell_type": "code",
   "id": "be4fc4bdd93c73d4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:37:05.927039Z",
     "start_time": "2025-07-09T12:37:05.923827Z"
    }
   },
   "source": [
    "import torch\n",
    "\n",
    "# 初始化位置编码矩阵\n",
    "pe = torch.zeros(seq_len, d_model)\n",
    "pe"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 23
  },
  {
   "cell_type": "code",
   "id": "2d1ed94fd5d28bad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:37:08.424494Z",
     "start_time": "2025-07-09T12:37:08.420411Z"
    }
   },
   "source": [
    "# 生成pos\n",
    "pos = torch.arange(0, seq_len)\n",
    "pos"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 24
  },
  {
   "cell_type": "code",
   "id": "b065ff6d18ecd049",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:39:45.928172Z",
     "start_time": "2025-07-09T12:39:45.923507Z"
    }
   },
   "source": [
    "# 偶数列\n",
    "pe[:, 0] = torch.sin(pos / (10000 ** (2 * 0 / d_model)))\n",
    "pe[:, 2] = torch.sin(pos / (10000 ** (2 * 1 / d_model)))\n",
    "pe[:, 4] = torch.sin(pos / (10000 ** (2 * 2 / d_model)))\n",
    "pe[:, 6] = torch.sin(pos / (10000 ** (2 * 3 / d_model)))\n",
    "pe"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.8415, 0.0000, 0.0998, 0.0000, 0.0100, 0.0000, 0.0010, 0.0000],\n",
       "        [0.9093, 0.0000, 0.1987, 0.0000, 0.0200, 0.0000, 0.0020, 0.0000],\n",
       "        [0.1411, 0.0000, 0.2955, 0.0000, 0.0300, 0.0000, 0.0030, 0.0000]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 28
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:41:17.072913Z",
     "start_time": "2025-07-09T12:41:17.064422Z"
    }
   },
   "source": [
    "# 奇数列\n",
    "pe[:, 1] = torch.cos(pos / (10000 ** (2 * 0 / d_model)))\n",
    "pe[:, 3] = torch.cos(pos / (10000 ** (2 * 1 / d_model)))\n",
    "pe[:, 5] = torch.cos(pos / (10000 ** (2 * 2 / d_model)))\n",
    "pe[:, 7] = torch.cos(pos / (10000 ** (2 * 3 / d_model)))\n",
    "pe"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0000e+00,  1.0000e+00,  0.0000e+00,  1.0000e+00,  0.0000e+00,\n",
       "          1.0000e+00,  0.0000e+00,  1.0000e+00],\n",
       "        [ 8.4147e-01,  5.4030e-01,  9.9833e-02,  9.9500e-01,  9.9998e-03,\n",
       "          9.9995e-01,  1.0000e-03,  1.0000e+00],\n",
       "        [ 9.0930e-01, -4.1615e-01,  1.9867e-01,  9.8007e-01,  1.9999e-02,\n",
       "          9.9980e-01,  2.0000e-03,  1.0000e+00],\n",
       "        [ 1.4112e-01, -9.8999e-01,  2.9552e-01,  9.5534e-01,  2.9995e-02,\n",
       "          9.9955e-01,  3.0000e-03,  1.0000e+00]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:48:15.536328Z",
     "start_time": "2025-07-09T12:48:15.531621Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "import math\n",
    "\n",
    "seq_len = 4  # 序列长度\n",
    "d_model = 8  # 词向量长度\n",
    "pe = torch.zeros(seq_len, d_model)\n",
    "position = torch.arange(0, seq_len, dtype=torch.float).unsqueeze(1)\n",
    "div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "pe[:, 0::2] = torch.sin(position * div_term)\n",
    "pe[:, 1::2] = torch.cos(position * div_term)\n",
    "pe"
   ],
   "id": "ffada3f5543149f",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0000e+00,  1.0000e+00,  0.0000e+00,  1.0000e+00,  0.0000e+00,\n",
       "          1.0000e+00,  0.0000e+00,  1.0000e+00],\n",
       "        [ 8.4147e-01,  5.4030e-01,  9.9833e-02,  9.9500e-01,  9.9998e-03,\n",
       "          9.9995e-01,  1.0000e-03,  1.0000e+00],\n",
       "        [ 9.0930e-01, -4.1615e-01,  1.9867e-01,  9.8007e-01,  1.9999e-02,\n",
       "          9.9980e-01,  2.0000e-03,  1.0000e+00],\n",
       "        [ 1.4112e-01, -9.8999e-01,  2.9552e-01,  9.5534e-01,  2.9995e-02,\n",
       "          9.9955e-01,  3.0000e-03,  1.0000e+00]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 33
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:49:00.854278Z",
     "start_time": "2025-07-09T12:49:00.837949Z"
    }
   },
   "cell_type": "code",
   "source": "x + pe",
   "id": "43a62058967f45f6",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.8978,  1.4811,  0.1846,  1.3993,  0.8489,  1.9601,  0.0934,  1.2663],\n",
       "        [ 1.1986,  0.9267,  0.6494,  1.3896,  0.3661,  1.3167,  0.5581,  1.4940],\n",
       "        [ 1.5819, -0.0623,  1.1967,  1.1462,  0.4005,  1.3073,  0.2352,  1.5693],\n",
       "        [ 0.8103, -0.0179,  1.2134,  1.7622,  0.3278,  1.5633,  0.2310,  1.9227]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 34
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:50:06.168436Z",
     "start_time": "2025-07-09T12:50:06.161480Z"
    }
   },
   "cell_type": "code",
   "source": [
    "seq_len = 256  # 序列长度\n",
    "d_model = 8  # 词向量长度\n",
    "pe = torch.zeros(seq_len, d_model)\n",
    "position = torch.arange(0, seq_len, dtype=torch.float).unsqueeze(1)\n",
    "div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "pe[:, 0::2] = torch.sin(position * div_term)\n",
    "pe[:, 1::2] = torch.cos(position * div_term)\n",
    "pe"
   ],
   "id": "b45deddc4639f56e",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0000e+00,  1.0000e+00,  0.0000e+00,  ...,  1.0000e+00,\n",
       "          0.0000e+00,  1.0000e+00],\n",
       "        [ 8.4147e-01,  5.4030e-01,  9.9833e-02,  ...,  9.9995e-01,\n",
       "          1.0000e-03,  1.0000e+00],\n",
       "        [ 9.0930e-01, -4.1615e-01,  1.9867e-01,  ...,  9.9980e-01,\n",
       "          2.0000e-03,  1.0000e+00],\n",
       "        ...,\n",
       "        [ 9.9482e-01, -1.0162e-01,  1.6648e-01,  ..., -8.1873e-01,\n",
       "          2.5031e-01,  9.6817e-01],\n",
       "        [ 4.5200e-01, -8.9202e-01,  2.6409e-01,  ..., -8.2444e-01,\n",
       "          2.5128e-01,  9.6792e-01],\n",
       "        [-5.0639e-01, -8.6230e-01,  3.5906e-01,  ..., -8.3005e-01,\n",
       "          2.5225e-01,  9.6766e-01]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 35
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-09T12:50:25.689072Z",
     "start_time": "2025-07-09T12:50:25.684483Z"
    }
   },
   "cell_type": "code",
   "source": "x + pe[:x.size(0), :]",
   "id": "75090a9cb183b0e6",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.8978,  1.4811,  0.1846,  1.3993,  0.8489,  1.9601,  0.0934,  1.2663],\n",
       "        [ 1.1986,  0.9267,  0.6494,  1.3896,  0.3661,  1.3167,  0.5581,  1.4940],\n",
       "        [ 1.5819, -0.0623,  1.1967,  1.1462,  0.4005,  1.3073,  0.2352,  1.5693],\n",
       "        [ 0.8103, -0.0179,  1.2134,  1.7622,  0.3278,  1.5633,  0.2310,  1.9227]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 36
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
